# -*- coding: utf-8 -*-
import scrapy


class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['douban.com']
    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        print(response.request.headers['User-Agent'])
        data = response.xpath('//*[@id="content"]/div/div[1]/ol/li/div/div[2]')

        for i in data:
            item = {}
            item['name'] = i.xpath('./div[1]/a/span[1]/text()').get()
            # print(item['name'])
            yield item

        url = response.xpath('//*[@id="content"]/div/div[1]/div[2]/span[3]/a/@href').get()
        if url != None:
            url = response.urljoin(url)
            print(url)
            yield scrapy.Request(url=url, callback=self.parse)
